{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "DataAnalysis"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from tools import show_subtitle\n",
    "%matplotlib inline"
   ]
  },
  {
   "source": [
    "# Chap4 编写结构化的程序\n",
    "1.  怎样才能写出结构良好，可读性强的程序，从而方便重用？\n",
    "2.  基本的结构块，例如：循环、函数和赋值是如何执行的？\n",
    "3.  Python 编程的陷阱还有哪些，如何避免它们？"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "## 4.2 序列（Sequences）(P147)\n",
    "序列对象共有三种（字符串、链表、元组)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "'t1 == t2'=  True\nt1[0]=  walk\nt2[1:]=  ('fem', 3)\nlen(t1)=  3\n"
     ]
    }
   ],
   "source": [
    "# 元组的定义与使用\n",
    "t1 = 'walk', 'fem', 3  # 定义元组的标准方法\n",
    "t2 = ('walk', 'fem', 3)  # 括号并非定义元组的标准方法，括号是Python语法的一般功能，是用于分组的\n",
    "print(\"'t1 == t2'= \", t1 == t2)\n",
    "print(\"t1[0]= \", t1[0])\n",
    "print(\"t2[1:]= \", t2[1:])\n",
    "print(\"len(t1)= \", len(t1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "'t3 == t4'=  True\n"
     ]
    }
   ],
   "source": [
    "t3 = ['a', 'b', 'c'], 'a', 'b', 'c'\n",
    "t4 = (['a', 'b', 'c'], 'a', 'b', 'c')\n",
    "print(\"'t3 == t4'= \",t3 == t4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "()\n"
     ]
    }
   ],
   "source": [
    "emtpy_tuple = ()  # 空元组的定义\n",
    "print(emtpy_tuple)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "('love',)\nlove\n"
     ]
    }
   ],
   "source": [
    "single_tuple = 'love',  # 单个元素的元组的定义\n",
    "print(single_tuple)\n",
    "print(single_tuple[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "raw[2]=t, text[3]=the, pair[1]=turned\nraw[-3:]=ute, text[-3:]=['off', 'the', 'spectroroute'], pair[-3:]=(6, 'turned')\nlen(raw)=29, len(text)=5, len(pair)=2\n"
     ]
    }
   ],
   "source": [
    "# spectroroute这个单词查不到\n",
    "raw = 'I turned off the spectroroute'  # 字符串\n",
    "text = ['I', 'turned', 'off', 'the', 'spectroroute']  # 链表\n",
    "pair = (6, 'turned')  # 元组\n",
    "print('raw[2]={}, text[3]={}, pair[1]={}'.format(raw[2], text[3], pair[1]))\n",
    "print('raw[-3:]={}, text[-3:]={}, pair[-3:]={}'.format(raw[-3:], text[-3:], pair[-3:]))\n",
    "print('len(raw)={}, len(text)={}, len(pair)={}'.format(len(raw), len(text), len(pair)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "sets=  {'d', 'h', ' ', 't', 'e', 'p', 'f', 'r', 'n', 'c', 's', 'o', 'u', 'I'}\n",
      "len(set(raw))=  14\n",
      "'f' in sets=  True\n",
      "lists=  ['d', 'h', ' ', 't', 'e', 'p', 'f', 'r', 'n', 'c', 's', 'o', 'u', 'I']\n",
      "len(lists)=  14\n",
      "'f' in lists=  True\n",
      "sorted(set(text))=  ['I', 'off', 'spectroroute', 'the', 'turned']\n"
     ]
    }
   ],
   "source": [
    "# 定义集合\n",
    "# 集合不可以索引访问\n",
    "sets = set(raw)\n",
    "print(\"sets= \",sets)\n",
    "print(\"len(set(raw))= \", len(sets))\n",
    "print(\"'f' in sets= \", 'f' in sets)\n",
    "\n",
    "lists = list(sets)\n",
    "print(\"lists= \", lists)\n",
    "print(\"len(lists)= \", len(sets))\n",
    "print(\"'f' in lists= \", 'f' in lists)\n",
    "\n",
    "sets = set(text)\n",
    "print(\"sorted(set(text))= \", sorted(sets))"
   ]
  },
  {
   "source": [
    "### 4.2.1 序列类型上的操作(P148)\n",
    "表4-1：遍历序列的各种方式"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "set(raw)=  {'d', ' ', 'e', 'l', 'R', 'w', 'y', 'r', ',', 'o'}\n",
      "list(raw)=  ['R', 'e', 'd', ' ', 'l', 'o', 'r', 'r', 'y', ',', ' ', 'y', 'e', 'l', 'l', 'o', 'w', ' ', 'l', 'o', 'r', 'r', 'y', ',', ' ', 'r', 'e', 'd', ' ', 'l', 'o', 'r', 'r', 'y', ',', ' ', 'y', 'e', 'l', 'l', 'o', 'w', ' ', 'l', 'o', 'r', 'r', 'y']\n"
     ]
    }
   ],
   "source": [
    "raw = 'Red lorry, yellow lorry, red lorry, yellow lorry'\n",
    "print(\"set(raw)= \", set(raw))  # 不能出现重复的元素\n",
    "print(\"list(raw)= \", list(raw))  # 可以出现重复的元素"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      " lorry      , yellow    Red    red \n",
      "     4      3      2      1      1 \n",
      "sorted(fdist)=  [',', 'Red', 'lorry', 'red', 'yellow']\n",
      "lorry: 4\n",
      ",: 3\n",
      "yellow: 2\n",
      "Red: 1\n",
      "red: 1\n"
     ]
    }
   ],
   "source": [
    "from nltk import word_tokenize\n",
    "\n",
    "text = word_tokenize(raw)  # 分词\n",
    "fdist = nltk.FreqDist(text)  # 词频统计\n",
    "fdist.tabulate()\n",
    "print(\"sorted(fdist)= \",sorted(fdist))\n",
    "for key in fdist:\n",
    "    print(key + ':', fdist[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "word_tokenize(raw)=  ['I', 'turned', 'off', 'the', 'spectroroute']\n"
     ]
    }
   ],
   "source": [
    "raw = 'I turned off the spectroroute'\n",
    "text = word_tokenize(raw)\n",
    "print(\"word_tokenize(raw)= \", text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "words=  ['I', 'turned', 'the', 'spectroroute', 'off']\n"
     ]
    }
   ],
   "source": [
    "words = ['I', 'turned', 'off', 'the', 'spectroroute']\n",
    "# 利用元组批量赋值\n",
    "words[2], words[3], words[4] = words[3], words[4], words[2]\n",
    "print(\"words= \", words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = ['I', 'turned', 'off', 'the', 'spectroroute', '!']\n",
    "tags = ['noun', 'verb', 'prep', 'det', 'noun']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "list(zip(words, tags))=  [('I', 'noun'), ('turned', 'verb'), ('off', 'prep'), ('the', 'det'), ('spectroroute', 'noun')]\nlist(zip(words, tags, tags))=  [('I', 'noun', 'noun'), ('turned', 'verb', 'verb'), ('off', 'prep', 'prep'), ('the', 'det', 'det'), ('spectroroute', 'noun', 'noun')]\n"
     ]
    }
   ],
   "source": [
    "# zip() 取出两个或者两个以上的序列中的项目，将它们“压缩”打包成单个的配对链表\n",
    "# 数目不匹配的部分直接丢弃，例如：words比赛tags多一个，就直接丢弃了。\n",
    "print(\"list(zip(words, tags))= \", list(zip(words, tags)))\n",
    "print(\"list(zip(words, tags, tags))= \", list(zip(words, tags, tags)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "list(enumerate(words))=  [(0, 'I'), (1, 'turned'), (2, 'off'), (3, 'the'), (4, 'spectroroute'), (5, '!')]\n"
     ]
    }
   ],
   "source": [
    "# enumerate() 返回一个包含索引及索引处所在项目的配对。\n",
    "print(\"list(enumerate(words))= \",list(enumerate(words)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "a:0\tb:I\na:1\tb:turned\na:2\tb:off\na:3\tb:the\na:4\tb:spectroroute\na:5\tb:!\n"
     ]
    }
   ],
   "source": [
    "for a, b in enumerate(words):\n",
    "    print('a:{}\\tb:{}'.format(a, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "len(training_data) / len(test_data)=  9.0\n"
     ]
    }
   ],
   "source": [
    "# 分割数据集为（训练集+测试集)\n",
    "text = nltk.corpus.nps_chat.words()\n",
    "cut = int(0.9 * len(text))\n",
    "training_data, test_data = text[:cut], text[cut:]\n",
    "print(\"len(training_data) / len(test_data)= \",len(training_data) / len(test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "words=  ['I', 'turned', 'off', 'the', 'spectroroute']\nwordlens=  [(1, 'I'), (6, 'turned'), (3, 'off'), (3, 'the'), (12, 'spectroroute')]\n"
     ]
    }
   ],
   "source": [
    "# 使用split()函数分词\n",
    "raw = 'I turned off the spectroroute'\n",
    "words = raw.split()\n",
    "print(\"words= \", words)\n",
    "wordlens = [(len(word), word) for word in words]\n",
    "print(\"wordlens= \", wordlens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "--------------- >wordlens.sort()< ---------------\nwordlens=  [(1, 'I'), (3, 'off'), (3, 'the'), (6, 'turned'), (12, 'spectroroute')]\n"
     ]
    }
   ],
   "source": [
    "wordlens.sort()\n",
    "show_subtitle(\"wordlens.sort()\")\n",
    "print(\"wordlens= \", wordlens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "--------------- >wordlens.reverse()< ---------------\nwordlens=  [(12, 'spectroroute'), (6, 'turned'), (3, 'the'), (3, 'off'), (1, 'I')]\n"
     ]
    }
   ],
   "source": [
    "wordlens.reverse()\n",
    "show_subtitle(\"wordlens.reverse()\")\n",
    "print(\"wordlens= \", wordlens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "--------------- >wordlens.pop()< ---------------\nwordlens=  [(12, 'spectroroute'), (6, 'turned'), (3, 'the'), (3, 'off')]\n"
     ]
    }
   ],
   "source": [
    "wordlens.pop()\n",
    "show_subtitle(\"wordlens.pop()\")\n",
    "print(\"wordlens= \", wordlens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "' '.join(w for (_, w) in wordlens)=  spectroroute turned the off\n"
     ]
    }
   ],
   "source": [
    "print(\"' '.join(w for (_, w) in wordlens)= \",\n",
    "      ' '.join(\n",
    "              w\n",
    "              for (_, w) in wordlens\n",
    "      )\n",
    "      )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "lexicon=  [('the', 'det', ['Di:', 'D@']), ('off', 'prep', ['Qf', 'O:f'])]\n"
     ]
    }
   ],
   "source": [
    "# 元组是不可修改的，而链表是可以修改的\n",
    "lexicon = [('the', 'det', ['Di:', 'D@']), ('off', 'prep', ['Qf', 'O:f'])]\n",
    "print(\"lexicon= \", lexicon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "--------------- >lexicon.sort()< ---------------\nlexicon=  [('off', 'prep', ['Qf', 'O:f']), ('the', 'det', ['Di:', 'D@'])]\n"
     ]
    }
   ],
   "source": [
    "lexicon.sort()\n",
    "show_subtitle(\"lexicon.sort()\")\n",
    "print(\"lexicon= \", lexicon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "--------------- >lexicon[1] = ('turned', 'VBD', ['t3:nd', 't3`nd'])< ---------------\nlexicon=  [('off', 'prep', ['Qf', 'O:f']), ('turned', 'VBD', ['t3:nd', 't3`nd'])]\n"
     ]
    }
   ],
   "source": [
    "lexicon[1] = ('turned', 'VBD', ['t3:nd', 't3`nd'])\n",
    "show_subtitle(\"lexicon[1] = ('turned', 'VBD', ['t3:nd', 't3`nd'])\")\n",
    "print(\"lexicon= \", lexicon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "--------------- >del lexicon[0]< ---------------\nlexicon=  [('turned', 'VBD', ['t3:nd', 't3`nd'])]\n"
     ]
    }
   ],
   "source": [
    "del lexicon[0]\n",
    "show_subtitle(\"del lexicon[0]\")\n",
    "print(\"lexicon= \", lexicon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 链表转换成元组后，下面的操作都不可执行\n",
    "lexicon = tuple(lexicon)\n",
    "# lexicon.sort()\n",
    "# lexicon[1] = ('turned', 'VBD', ['t3:nd', 't3`nd'])\n",
    "# del lexicon[0]"
   ]
  },
  {
   "source": [
    "使用元组还是链表主要还是取决于项目的内容是否与它的位置相关"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "### 4.2.2 产生器表达式(P151)\n",
    "使用列表推导式方便处理文本"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "text=  \"When I use a word,\" Humpty Dumpty said in rather a scornful tone, \n\"it means just what I choose it to mean - neither more nor less.\"\n"
     ]
    }
   ],
   "source": [
    "text = '''\"When I use a word,\" Humpty Dumpty said in rather a scornful tone, \n",
    "\"it means just what I choose it to mean - neither more nor less.\"'''\n",
    "print(\"text= \", text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "words=  ['``', 'when', 'i', 'use', 'a', 'word', ',', \"''\", 'humpty', 'dumpty', 'said', 'in', 'rather', 'a', 'scornful', 'tone', ',', \"''\", 'it', 'means', 'just', 'what', 'i', 'choose', 'it', 'to', 'mean', '-', 'neither', 'more', 'nor', 'less', '.', \"''\"]\n"
     ]
    }
   ],
   "source": [
    "words = [\n",
    "        w.lower()\n",
    "        for w in nltk.word_tokenize(text)\n",
    "]\n",
    "print(\"words= \", words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "words[0]=  ``\nwords[7]=  ''\n'words[0] == words[7]'=  False\n"
     ]
    }
   ],
   "source": [
    "print(\"words[0]= \", words[0])\n",
    "print(\"words[7]= \", words[7])\n",
    "# ToDo: 句子开头和结尾的双引号是有区别的\n",
    "print(\"'words[0] == words[7]'= \", words[0] == words[7])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "max(words)=  word\nmax_words=  word\n"
     ]
    }
   ],
   "source": [
    "# 这两种方式都会先生成链表对象，再调用 max() 函数，\n",
    "# 会占用大量的存储空间\n",
    "print(\"max(words)= \", max(words))\n",
    "max_words = max([\n",
    "        w.lower()\n",
    "        for w in nltk.word_tokenize(text)\n",
    "])\n",
    "print(\"max_words= \", max_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "max_words=  word\n"
     ]
    }
   ],
   "source": [
    "# max()函数调用时使用了产生器表达式，\n",
    "# 不仅仅是标记方便，即不仅仅是省略了方括号[]，\n",
    "# 还利用数据流输入到调用函数中，避免先生成链表对象，也就不会占用过多的存储空间\n",
    "max_words = max(\n",
    "        w.lower()\n",
    "        for w in nltk.word_tokenize(text)\n",
    ")\n",
    "print(\"max_words= \", max_words)"
   ]
  }
 ]
}